package forumTransform.forumTransformID;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.TreeSet;

import org.apache.commons.lang3.StringEscapeUtils;
import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.util.EntityUtils;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import com.google.gson.Gson;


public class Utils {

	public static TreeSet<String> htmlNumbers =new TreeSet<String>();
	
	public static Long noRequests = 0L;
	
	public static void populateHtmlNumbers(){
    	htmlNumbers.add("&#8211;");
    	htmlNumbers.add("&#8212;");
    	htmlNumbers.add("&#8216;");
    	htmlNumbers.add("&#8217;");
    	htmlNumbers.add("&#8218;");
    	htmlNumbers.add("&#8220;");
    	htmlNumbers.add("&#8221;");
    	htmlNumbers.add("&#8222;");
    	htmlNumbers.add("&#8224;");
    	htmlNumbers.add("&#8225;");
    	htmlNumbers.add("&#8226;");
    	htmlNumbers.add("&#8230;");
    	htmlNumbers.add("&#8240;");
    	htmlNumbers.add("&#8364;");
    	htmlNumbers.add("&#8242;");
    	htmlNumbers.add("&#8482;");
    	htmlNumbers.add("&#8482;");
    	htmlNumbers.add("&rdquo;");
    	htmlNumbers.add("&ldquo;");
    	htmlNumbers.add("&rsquo;");
    	htmlNumbers.add("&lsquo;");
    	htmlNumbers.add("&ndash;");
    	htmlNumbers.add("&mdash;");
    	htmlNumbers.add("&hellip;");
    	
    	
	}
	
	public static ArrayList<Utterance> processMessagesList(Elements messages, String topicId){
		ArrayList<Utterance> result = new ArrayList<Utterance>();
		int i=0;
		for (Iterator<Element> it = messages.iterator(); it.hasNext();) {
			i++;
			Element el = (Element) it.next();
			Utterance u = new Utterance(i, topicId);
			processMessage(el,u);
			result.add(u);
		}
		
		return result;
	}
	
	public static String escapeText(String s){
		s = s.replaceAll("\\<[^>]*>","");
		for (Iterator<String> it = Utils.htmlNumbers.iterator(); it.hasNext();) {
			String token = (String) it.next();
			s = s.replaceAll(token, "");
		}
		s = s.replaceAll("&egrave;","e");
		s = s.replaceAll("&eacute;","e");
//		s = s.replaceAll("&", "&amp;");
//        s = s.replaceAll("\"", "&quot;");
//        s = s.replaceAll("'", "&apos;");
//        s = s.replaceAll("<", "&lt;");
//        s = s.replaceAll(">", "&gt;");
		s = s.replaceAll("&nbsp;", " ");
		s	= StringEscapeUtils.escapeHtml4(s);
        return s;
        
    }
	
//	for (ListIterator iterator = list.listIterator(list.size()); iterator.hasPrevious();) {
// 	  final Object listElement = iterator.previous();
//  }

	
	public static void processMessage(Element m, Utterance u){
		String id = m.attr("id");
		String subject = m.getElementById(id+"-subject").text();
		String author = m.getElementById(id+"-postedBy").text();
		String time = m.getElementById(id+"-postedAt").text();
		String content = m.getElementById(id+"-detail").text();
		
		int level = 0;
		Element tbody = m.getElementsByTag("tbody").get(0);
		Element tr = tbody.getElementsByTag("tr").get(0);
		Element td = tr.getElementsByTag("td").get(0);
		String style = td.attr("style").trim();
		level = Integer.parseInt(style.substring(style.lastIndexOf(':')+1,style.lastIndexOf("px;")))/25;
		
		u.setMsgId(id);
		u.setSubject(subject);
		u.setUserId(author);
		u.setUtteranceTime(time);
		u.setText(content);
		u.setLevel(level);
		System.out.println(td.attr("style").trim()+"#"+level+"#");
	//	System.out.println(m);
		
	}
    public static List<String> getTopicFiles() {
        return getFiles(".html", "in/");
    }
    
    public static List<String> getCSVFiles() {
        return getFiles(".csv", "in-csv/");
    }
    
    public static List<String> getFiles(String sufix, String dir) {
        List<String> retList    = new ArrayList<String>();
        File f                    = new File (dir + ".");
        String [] list            = f.list();
       
        if ( list != null && list.length > 0) {
            for (int i = 0; i < list.length; i++) {
                String filename = list[i];
                if ( filename.endsWith(sufix) ) {
                    retList.add(filename);
                }
               
            }
        }
       
        return retList;
    }
	
	public static ByteArrayOutputStream crawlURL(String url) throws ClientProtocolException, IOException{
		//OutputStream result = null;
		ByteArrayOutputStream result = new ByteArrayOutputStream();
        DefaultHttpClient httpclient = new DefaultHttpClient();
        HttpGet httpGet = new HttpGet(url);
        HttpResponse response1 = httpclient.execute(httpGet);
        try {
            System.out.println((++noRequests) +". request: "+url+" :"+response1.getStatusLine());
            HttpEntity entity1 = response1.getEntity();
            entity1.writeTo(result);
            EntityUtils.consume(entity1);
        } finally {
            httpGet.releaseConnection();
        }
		return result;
	
	}
	
	public static WordpressJSON createBlog(String url,boolean isComment){
		String newurl = url;
		if(newurl.startsWith("http://"))
			newurl = newurl.replaceAll("http://", "");
		if(newurl.startsWith("wwww"))
			newurl = newurl.replaceAll("www", "");
		if(newurl.endsWith("/"))
			newurl = newurl.substring(0, newurl.lastIndexOf("/"));
				
		
		WordpressJSON wJson = null;
		try {
			if(isComment)
				wJson = getInfoComments2(newurl,1,0);
			else wJson = getInfo(newurl,1,0);
		} catch (ClientProtocolException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
		wJson.setBlogId(newurl);
		if(wJson.getPosts()!=null)
			wJson.getPosts().clear();
		if(isComment)
			System.out.println("Blog: "+newurl+" with "+wJson.getFound()+" comments created");
		else
			System.out.println("Blog: "+newurl+" with "+wJson.getFound()+" posts created");
		return wJson;
	}

	public static WordpressJSON getInfo(String blogURL, int number, int offset) throws ClientProtocolException, IOException{
	   	 ByteArrayOutputStream os = Utils.crawlURL("https://public-api.wordpress.com/rest/v1/sites/"+blogURL+"/posts/?number="+number+"&pretty=1&offset="+offset);
	   	 String s = new String(os.toByteArray(),"UTF-8");
	   	 WordpressJSON wJson = new Gson().fromJson(s, WordpressJSON.class);
//	   	WPost post = wJson.getPostById(2064D);
//	   	if(post!=null){
//	   		System.out.println(Utils.escapeText(post.getContent()));
//	   	}
	   	 System.out.println("Blog: "+blogURL+" number:"+number+" offset:"+offset);
	   	 return wJson;
	}


	public static WordpressJSON createBlogForComments(String url, String postId){
		WordpressJSON wJson = null;
		try {
			wJson = getInfoComments(url,postId, 1,0);
		} catch (ClientProtocolException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
		wJson.setBlogId(url);
		if(wJson.getPosts()!=null) 
			wJson.getPosts().clear();
		System.out.println("Blog: "+url+"with "+wJson.getFound()+" comments created");
		return wJson;
	}
	
	
	//https://public-api.wordpress.com/rest/v1/sites/rwer.wordpress.com/posts/12818/replies/?pretty=1
	public static WordpressJSON getInfoComments(String blogURL, String postId, int number, int offset) throws ClientProtocolException, IOException{
	   	 ByteArrayOutputStream os = Utils.crawlURL("https://public-api.wordpress.com/rest/v1/sites/"+blogURL+"/posts/"+postId+"/replies/?number="+number+"&pretty=1&offset="+offset);
	   	 String s = new String(os.toByteArray(),"UTF-8");
	   	 WordpressJSON wJson = new Gson().fromJson(s, WordpressJSON.class);
	   	 System.out.println("Blog: "+blogURL+" postId:"+postId+" number:"+number+" offset:"+offset);
	   	 return wJson;
	}
	
	public static WordpressJSON getInfoComments2(String blogURL, int number, int offset) throws ClientProtocolException, IOException{
	   	 ByteArrayOutputStream os = Utils.crawlURL("https://public-api.wordpress.com/rest/v1/sites/"+blogURL+"/comments/?number="+number+"&pretty=1&offset="+offset);
	   	 String s = new String(os.toByteArray(),"UTF-8");
	   	 WordpressJSON wJson = new Gson().fromJson(s, WordpressJSON.class);
	   	 System.out.println("Blog: "+blogURL+" number:"+number+" offset:"+offset);
	   	 return wJson;
	}
	
	
	/*
	 * BLOGSPOT methods
	 */
	
	public static BlogspotJSON getBInfo(String blogURL, int number, int offset) throws ClientProtocolException, IOException{
	   	 ByteArrayOutputStream os = Utils.crawlURL("https://public-api.wordpress.com/rest/v1/sites/"+blogURL+"/posts/?number="+number+"&pretty=1&offset="+offset);
	   	 String s = new String(os.toByteArray(),"UTF-8");
	   	BlogspotJSON wJson = new Gson().fromJson(s, BlogspotJSON.class);

	   	 System.out.println("Blog: "+blogURL+" number:"+number+" offset:"+offset);
	   	 return wJson;
	}


	
	
	public static BlogspotJSON createBBlogForComments(String url, String postId){
		BlogspotJSON wJson = null;
		try {
			wJson = getBInfoComments(url,postId, 1,0);
		} catch (ClientProtocolException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
		wJson.setBlogId(url);
		if(wJson.getItems()!=null) 
			wJson.getItems().clear();
		System.out.println("Blog: "+url+"with "+wJson.getFound()+" comments created");
		return wJson;
	}
	
	
	public static BlogspotJSON getBInfoComments(String blogURL, String postId, int number, int offset) throws ClientProtocolException, IOException{
	   	 ByteArrayOutputStream os = Utils.crawlURL("https://public-api.wordpress.com/rest/v1/sites/"+blogURL+"/posts/"+postId+"/replies/?number="+number+"&pretty=1&offset="+offset);
	   	 String s = new String(os.toByteArray(),"UTF-8");
	   	BlogspotJSON wJson = new Gson().fromJson(s, BlogspotJSON.class);
	   	 System.out.println("Blog: "+blogURL+" postId:"+postId+" number:"+number+" offset:"+offset);
	   	 return wJson;
	}
	
	
	public static BlogspotJSON createBBlog(String url,boolean isComment){
		String newurl = url;
		if(newurl.startsWith("http://"))
			newurl = newurl.replaceAll("http://", "");
		if(newurl.startsWith("wwww"))
			newurl = newurl.replaceAll("www", "");
		if(newurl.endsWith("/"))
			newurl = newurl.substring(0, newurl.lastIndexOf("/"));
				
		
		BlogspotJSON wJson = null;
		try {
			if(isComment)
				wJson = getBInfoComments2(newurl,1,0);
			else wJson = getBInfo(newurl,1,0);
		} catch (ClientProtocolException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
		wJson.setBlogId(newurl);
		if(wJson.getItems()!=null)
			wJson.getItems().clear();
		System.out.println("Blog: "+newurl+"with "+wJson.getFound()+" posts created");
		return wJson;
	}

	public static BlogspotJSON getBInfoComments2(String blogURL, int number, int offset) throws ClientProtocolException, IOException{
	   	 ByteArrayOutputStream os = Utils.crawlURL("https://public-api.wordpress.com/rest/v1/sites/"+blogURL+"/comments/?number="+number+"&pretty=1&offset="+offset);
	   	 String s = new String(os.toByteArray(),"UTF-8");
	   	BlogspotJSON wJson = new Gson().fromJson(s, BlogspotJSON.class);
	   	 System.out.println("Blog: "+blogURL+" number:"+number+" offset:"+offset);
	   	 return wJson;
	}

	public static List<BPost> getBPosts(String id) throws ClientProtocolException, IOException {
		String nextToken = null;
		List<BPost> items = new ArrayList<BPost>();
		BlogspotJSON bJson = Utils.getBPostsFromServer(id, nextToken);
		if(bJson == null || bJson.getItems() == null) return items;
		items.addAll(bJson.getItems());
		while(isValid(bJson.getNextPageToken()))
			 {
				bJson = Utils.getBPostsFromServer(id, bJson.getNextPageToken());
				if(bJson!=null && bJson.getItems()!=null)
					items.addAll(bJson.getItems());
			 }
		return items;
	}
	
	public static boolean isValid(String nextToken){
		if(nextToken ==null || "".compareTo(nextToken.trim())==0)
			return false;
		return true;
	}
	
	//https://www.googleapis.com/blogger/v3/blogs/29059245/posts?maxResults=20&key=
	//API_KEY= AIzaSyDfAzQeFwXL4kgNBT-CvwFUqOlTGaYZh7A
	private static BlogspotJSON getBPostsFromServer(String blogId, String pageToken) throws ClientProtocolException, IOException {
		String url = null;
		if(pageToken==null || "".compareTo(pageToken.trim())==0)
			url = "https://www.googleapis.com/blogger/v3/blogs/"+blogId+"/posts?maxResults=20&key="+Constants.BloggerAPI;
		else url = "https://www.googleapis.com/blogger/v3/blogs/"+blogId+"/posts?maxResults=20&pageToken="+pageToken+"&key="+Constants.BloggerAPI;
		ByteArrayOutputStream os = Utils.crawlURL(url);
		String s = new String(os.toByteArray(),"UTF-8");
		BlogspotJSON bJson = new Gson().fromJson(s, BlogspotJSON.class);
		return bJson;
	}

	//https://www.googleapis.com/blogger/v3/blogs/byurl?url=http%3A%2F%2Fscience-professor.blogspot.com&key=AIzaSyDfAzQeFwXL4kgNBT-CvwFUqOlTGaYZh7A
	public static BlogspotJSON getBInfoDetails(String url) throws ClientProtocolException, IOException{
		String blogUrl = url;
		if(blogUrl.startsWith("http://"))
			blogUrl = blogUrl.replaceAll("http://", "");
		if(blogUrl.startsWith("wwww"))
			blogUrl = blogUrl.replaceAll("www", "");
		if(blogUrl.endsWith("/"))
			blogUrl = blogUrl.substring(0, blogUrl.lastIndexOf("/"));
	   	ByteArrayOutputStream os = Utils.crawlURL("https://www.googleapis.com/blogger/v3/blogs/byurl?url=http%3A%2F%2F"+blogUrl+"&key="+Constants.BloggerAPI);
	   	 String s = new String(os.toByteArray(),"UTF-8");
	   	BlogspotJSON bJson = new Gson().fromJson(s, BlogspotJSON.class);
	   	bJson.setBlogId(blogUrl);
	   	 System.out.println("Blog: "+blogUrl+" posts:"+bJson.getPosts().getTotalItems()+" selfLink:"+bJson.getPosts().getTotalItems());
	   	 return bJson;
	}

	public static List<BComment> getBComments(String id, List<BPost> items) throws ClientProtocolException, IOException {
		// TODO Auto-generated method stub
		List<BComment> comItems = new ArrayList<BComment>();
		//int i = 0;
		for (BPost bPost : items) {
			String nextToken = null;
			//i++;
			//if(i>2) break;
			BlogspotCommentJSON bJson = Utils.getBCommentsFromServer(id, bPost.getId(), nextToken);
			if(bJson == null || bJson.getItems() == null) continue;
			
			comItems.addAll(bJson.getItems());
			while(isValid(bJson.getNextPageToken()))
				 {
					bJson = Utils.getBCommentsFromServer(id,bPost.getId(), bJson.getNextPageToken());
					if(bJson!=null && bJson.getItems()!=null)
						comItems.addAll(bJson.getItems());
				 }
		}
		return comItems;
	}

	//https://www.googleapis.com/blogger/v3/blogs/29059245/posts/7903140527192297607/comments?key=
	private static BlogspotCommentJSON getBCommentsFromServer(String blogId, String postId, String pageToken) throws ClientProtocolException, IOException {
		// TODO Auto-generated method stub
		String url = null;
		if(pageToken==null || "".compareTo(pageToken.trim())==0)
			url = "https://www.googleapis.com/blogger/v3/blogs/"+blogId+"/posts/"+postId+"/comments?maxResults=20&key="+Constants.BloggerAPI;
		else url = "https://www.googleapis.com/blogger/v3/blogs/"+blogId+"/posts/"+postId+"/comments?maxResults=20&pageToken="+pageToken+"&key="+Constants.BloggerAPI;
		ByteArrayOutputStream os = Utils.crawlURL(url);
		String s = new String(os.toByteArray(),"UTF-8");
		BlogspotCommentJSON bJson = new Gson().fromJson(s, BlogspotCommentJSON.class);
		return bJson;
	}
	
}
